package fm.cheyiwang.controller;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ConcurrentHashMultiset;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import fm.controller.BaseController;
import fm.dao.MongoBaseDao;
import fm.mongo.MongoTable;
import fm.nio.SemaphoreExecutor;
import fm.util.RequestHeader;
import fm.util.RequestUtils;
import fm.web.MediaTypes;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections.map.HashedMap;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;

import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;

/**
 * @修改人：CM
 * @修改时间：2017/2/26 21:12
 */
@Controller
@RequestMapping("/cheyiwang/data/col")
public class DataCollectionController extends BaseController {
    private static Boolean COLLECTING = true;
    private static final Logger LOGGER = LoggerFactory.getLogger(DataCollectionController.class);
    @Autowired
    private MongoBaseDao mongoDao;

    @ResponseBody
    @RequestMapping(value = "/product", method = RequestMethod.POST, name = MediaTypes.JSON_UTF_8)
    private Map colProductData(ModelMap modelMap) {
        Map result = new HashMap();
        COLLECTING = true;
        try {
            new Thread(new Runnable() {
                @Override
                public void run() {
                    int page = 1;
                    int count = 0;
                    while (COLLECTING) {
                        try {
                            Map param = new HashMap();
                            param.put("p", page + "");
                            param.put("vtrade", "0");
                            param.put("action", "FrontGetList");
                            param.put("st", "2");
                            String url = "http://www.2016927.com/Admin/Base/Product.ashx";
                            Collection<DBObject> datas = colData(url, param, "Status", "ok", "ProList");
                            count += datas.size();
                            if (CollectionUtils.isEmpty(datas)) {
                                COLLECTING = false;
                            } else {

                                String startProId = (String) datas.iterator().next().get("ProId");
                                Map<String, Object> queryParam = new HashedMap();
                                queryParam.put("ProId", startProId);
                                if (mongoDao.findOne(queryParam, DBObject.class, MongoTable.cyw_products) != null) {
                                    COLLECTING = false;
                                } else {
                                    mongoDao.insertAll(datas, MongoTable.cyw_products);
                                    LOGGER.info("当前抓取第{}页 共 {}条", page + "", datas.size() + "");
                                }
                            }

                        } catch (Exception ex) {
                            COLLECTING = false;
                            LOGGER.error("抓取数据发生错误,当前已抓取数据{}页 共 {}条", page + "", count + "", ex);
                        } finally {
                            page++;
                        }
                    }
                    LOGGER.info("抓取数据任务停止,当前已抓取数据{}页 共 {}条", page + "", count + "");
                }
            }).start();
            this.success(result);
        } catch (Exception ex) {
            this.failed(result, ex.getMessage());
            LOGGER.error("抓取数据发生错误", ex);
        }
        return result;
    }

    @ResponseBody
    @RequestMapping(value = "/purchase", method = RequestMethod.POST, name = MediaTypes.JSON_UTF_8)
    private Map colPurchaseData(ModelMap modelMap) {
        Map result = new HashMap();
        COLLECTING = true;
        try {
            new Thread(new Runnable() {
                @Override
                public void run() {
                    int page = 131;
                    int count = 0;
                    while (COLLECTING) {
                        try {
                            Map param = new HashMap();
                            param.put("p", page + "");
                            param.put("k", "");
                            param.put("action", "FrontGetList");
                            String url = "http://www.2016927.com/Admin/Business/PurchaseInfo.ashx";
                            Collection<DBObject> datas = colData(url, param, "Status", "1", "List");
                            count += datas.size();
                            if (CollectionUtils.isEmpty(datas)) {
                                COLLECTING = false;
                            } else {

                                String startPID = (String) datas.iterator().next().get("pid");
                                Map<String, Object> queryParam = new HashedMap();
                                queryParam.put("pid", startPID);
                                if (mongoDao.findOne(queryParam, DBObject.class, MongoTable.cyw_purchase) != null) {
                                    COLLECTING = false;
                                } else {
                                    mongoDao.insertAll(datas, MongoTable.cyw_purchase);
                                    LOGGER.info("当前抓取第{}页 共 {}条", page + "", datas.size() + "");
                                }
                            }

                        } catch (Exception ex) {
                            COLLECTING = false;
                            LOGGER.error("抓取数据发生错误,当前已抓取数据{}页 共 {}条", page + "", count + "", ex);
                        } finally {
                            page++;
                        }
                    }
                    LOGGER.info("抓取数据任务停止,当前已抓取数据{}页 共 {}条", page + "", count + "");
                }
            }).start();
            this.success(result);
        } catch (Exception ex) {
            this.failed(result, ex.getMessage());
            LOGGER.error("抓取数据发生错误", ex);
        }
        return result;
    }


    @ResponseBody
    @RequestMapping(value = "/task/stop", method = RequestMethod.POST, name = MediaTypes.JSON_UTF_8)
    private Map stopTask(ModelMap modelMap) {
        Map result = new HashMap();
        try {
            COLLECTING = false;
            this.success(result);
        } catch (Exception ex) {
            this.failed(result, ex.getMessage());
            LOGGER.error("抓取数据发生错误", ex);
        }
        return result;
    }


    private Collection<DBObject> colData(String url, Map param, String statusKey, String successValue, String listKey) throws Exception {
        String responseStr = RequestUtils.post(url, param);
        Map result = JSON.parseObject(responseStr, Map.class);
        Collection<DBObject> datas = new ArrayList<>();
        if (MapUtils.isNotEmpty(result) && successValue.equals(MapUtils.getString(result, statusKey, null).toLowerCase())) {
            Collection<JSONObject> data = (Collection) MapUtils.getObject(result, listKey);
            for (JSONObject obj : data) {
                DBObject dbObject = new BasicDBObject();
                dbObject.putAll(obj);
                datas.add(dbObject);
//                String proPic = obj.getString("ProPic");
//                if(StringUtils.isNotEmpty(proPic)){
//                    RequestUtils.httpRequestFile("http://www.2016927.com"+proPic,"D:\\项目\\wechat-website\\app\\target\\app-1.0.0\\resources\\cheyiwang\\"+proPic.replace("/","\\"));
//                }
            }
            if (CollectionUtils.isNotEmpty(datas)) {
                return datas;
            }
        }
        return CollectionUtils.EMPTY_COLLECTION;
    }

    final ConcurrentLinkedQueue<String> failedPicUri = new ConcurrentLinkedQueue<>();


    @ResponseBody
    @RequestMapping(value = "/img", method = RequestMethod.POST, name = MediaTypes.JSON_UTF_8)
    private Map colImg(ModelMap modelMap) {
        Map result = new HashMap();
        COLLECTING = true;
        try {
            new Thread(new Runnable() {
                @Override
                public void run() {
                    int count = 0;
                    final ConcurrentHashMultiset downloadedImage = ConcurrentHashMultiset.create();
                    try {
                        Collection<DBObject> products = mongoDao.findAll(DBObject.class, MongoTable.cyw_products);
                        Collection<DBObject> purchase = mongoDao.findAll(DBObject.class, MongoTable.cyw_purchase);

                        final CountDownLatch cdl = new CountDownLatch(products.size() + purchase.size());
                        SemaphoreExecutor executor = new SemaphoreExecutor(50, "picDownloadThread");
                        for (DBObject pro : products) {
                            final String pic = (String) pro.get("ProPic");
                            executor.execute(new Runnable() {
                                @Override
                                public void run() {
                                    try {
                                        if (!downloadedImage.contains(pic)) {
                                            if (StringUtils.isNotEmpty(pic)) {

                                                RequestHeader header = new RequestHeader();
                                                header.setAccept("image/webp,image/*,*/*;q=0.8");
                                                header.setAcceptEncoding("gzip, deflate, sdch");
                                                header.setAcceptLanguage("zh-CN,zh;q=0.8");
                                                header.setConnection("keep-alive");
                                                header.setCookie("__cfduid=d4c6454d129c7e518d6a449c386a9fc7e1488074382; yjs_use_ob=0; user_lng=118.10388605; user_lat=24.48923061; Hm_lvt_6d3936097458e84313a0e6d1aa60f655=1488074383,1488119584; Hm_lpvt_6d3936097458e84313a0e6d1aa60f655=1488122314");
                                                header.setHost("www.2016927.com");
                                                header.setReferer("http://www.2016927.com/ProList.aspx");
                                                RequestUtils.httpRequestFile("http://www.2016927.com" + pic,
                                                        "D:\\项目\\wechat-website\\app\\target\\app-1.0.0\\resources\\cheyiwang\\" + pic.replace("/", "\\"),
                                                        header);
                                            }
                                        } else {
                                            LOGGER.info("已经下载过的图片:{}", pic);
                                        }
                                    } catch (Exception ex) {
                                        LOGGER.error("获取图片：{}发生错误，已放入失败队列", pic, ex);
                                        failedPicUri.add(pic);
                                    } finally {
                                        downloadedImage.add(pic);
                                        cdl.countDown();
                                    }
                                }
                            });
                        }

                        for (DBObject pur : purchase) {
                            final String himg = (String) pur.get("himg");
                            final String pic1 = (String) pur.get("pic1");
                            final String pic2 = (String) pur.get("pic2");
                            final String pic3 = (String) pur.get("pic3");
                            final String pic4 = (String) pur.get("pic4");

                            final String pics[] = {himg, pic1, pic2, pic3, pic4};
                            executor.execute(new Runnable() {
                                @Override
                                public void run() {
                                    for (String pic : pics) {
                                        try {
                                            if (!downloadedImage.contains(pic)) {
                                                if (StringUtils.isNotEmpty(pic)) {
                                                    RequestHeader header = new RequestHeader();
                                                    header.setAccept("image/webp,image/*,*/*;q=0.8");
                                                    header.setAcceptEncoding("gzip, deflate, sdch");
                                                    header.setAcceptLanguage("zh-CN,zh;q=0.8");
                                                    header.setConnection("keep-alive");
                                                    header.setCookie("__cfduid=d4c6454d129c7e518d6a449c386a9fc7e1488074382; yjs_use_ob=0; user_lng=118.10388605; user_lat=24.48923061; Hm_lvt_6d3936097458e84313a0e6d1aa60f655=1488074383,1488119584; Hm_lpvt_6d3936097458e84313a0e6d1aa60f655=1488122314");
                                                    header.setHost("www.2016927.com");
                                                    header.setReferer("http://www.2016927.com/PurchaseList.aspx");
                                                    RequestUtils.httpRequestFile("http://www.2016927.com" + pic,
                                                            "D:\\项目\\wechat-website\\app\\target\\app-1.0.0\\resources\\cheyiwang\\" + pic.replace("/", "\\"),
                                                            header);
                                                }
                                            } else {
                                                LOGGER.info("已经下载过的图片:{}", pic);
                                            }
                                        } catch (Exception ex) {
                                            LOGGER.error("获取图片：{}发生错误，已放入失败队列", pic, ex);
                                            failedPicUri.add(pic);
                                        } finally {
                                            downloadedImage.add(pic);
                                            cdl.countDown();
                                        }
                                    }
                                }
                            });
                        }
                        cdl.await();
                    } catch (Exception ex) {
                        COLLECTING = false;
                        LOGGER.error("获取图片反生错误", ex);
                    } finally {
                        LOGGER.info("获取图片反生错误任务停止，共获取图片：{}张", count);
                    }
                }
            }).start();
            this.success(result);
        } catch (Exception ex) {
            this.failed(result, ex.getMessage());
            LOGGER.error("抓取数据发生错误", ex);
        }
        return result;
    }

}
